"""
This code is based on the tutorial on deeplearning.org


References:

    - textbooks: "Pattern Recognition and Machine Learning" -
                 Christopher M. Bishop, section 4.3.2

"""
__docformat__ = 'restructedtext en'

import cPickle
import gzip
import os
import sys
import time

import numpy
import nnutils

import theano
import theano.tensor as T


class SVM(object):
    def __init__(self, input, input_dropout,
                 n_in, n_out, numpy_rng, theano_rng, 
                 W=None,b=None,
                 dropout_rate = None,
                 L1rec = None, L2rec = None):

        # initialize with 0 the weights W as a matrix of shape (n_in, n_out)
        if W == None:
#            W_values = numpy.asarray( 
#            numpy_rng.normal(size=(n_in, n_out), scale=0.01), 
#            dtype=theano.config.floatX)
            W_values = nnutils.SparseInitialization(n_in, n_out, numpy_rng)
            self.W = theano.shared(value=W_values, name='W', borrow=True)

#             self.W = theano.shared(value=numpy.zeros((n_in, n_out),
#                                                  dtype=theano.config.floatX),
#                                 name='W', borrow=True)
        else:
            self.W = W
        # initialize the baises b as a vector of n_out 0s
        if b == None:
            self.b = theano.shared(value=numpy.zeros((n_out,),
                                                 dtype=theano.config.floatX),
                               name='b', borrow=True)
        else:
            self.b = b
        
        self.dropout_rate = dropout_rate
        # compute vector of class-membership probabilities in symbolic form
        if dropout_rate == None:
            self.p_y_given_x = T.dot(input, self.W) + self.b
            # compute prediction as class whose probability is maximal in
            # symbolic form
            self.y_pred = T.argmax(self.p_y_given_x, axis=1)
        else:
#            rate = theano.shared(numpy.asarray(dropout_rate,
#                                dtype=theano.config.floatX))
            dropout_h = T.dot(input_dropout \
            * T.cast(theano_rng.binomial(size=input_dropout.shape, n=1, p=1 - dropout_rate), 
                     dtype=theano.config.floatX),  \
            self.W)
            h = T.dot(input, self.W*(1-dropout_rate))
            self.p_y_given_x_dropout = dropout_h+self.b
            self.p_y_given_x = h+self.b
            self.y_pred = T.argmax(self.p_y_given_x, axis=1)

        

        # parameters of the model
        self.params = [self.W, self.b]

        self.L1rec = L1rec
        self.L2rec = L2rec
        
    def cost(self, y, C, L1_reg, L2_reg):
        # y.shape[0] is (symbolically) the number of rows in y, i.e.,
        # number of examples (call it n) in the minibatch
        # T.arange(y.shape[0]) is a symbolic vector which will contain
        # [0,1,2,... n-1] T.log(self.p_y_given_x) is a matrix of
        # Log-Probabilities (call it LP) with one row per example and
        # one column per class LP[T.arange(y.shape[0]),y] is a vector
        # v containing [LP[0,y[0]], LP[1,y[1]], LP[2,y[2]], ...,
        # LP[n-1,y[n-1]]] and T.mean(LP[T.arange(y.shape[0]),y]) is
        # the mean (across minibatch examples) of the elements in v,
        # i.e., the mean log-likelihood across the minibatch.
        if self.dropout_rate == None:  
            c = T.mean(T.sum(self.W**2, axis=0) + self.b**2) + \
            C*(T.mean(T.maximum(1-(self.p_y_given_x)[T.arange(y.shape[0]), y],0)**2 # positive examples
            + T.sum(T.maximum(1+self.p_y_given_x, 0)**2,axis=1)
            - T.maximum(1+(self.p_y_given_x)[T.arange(y.shape[0]), y],0)**2))
            + L1_reg*self.L1rec + L2_reg*self.L2rec
        else:
            c = T.mean(T.sum(self.W**2, axis=0) + self.b**2) + \
            C*(T.mean(T.maximum(1-(self.p_y_given_x_dropout)[T.arange(y.shape[0]), y],0)**2 # positive examples
            + T.sum(T.maximum(1+self.p_y_given_x_dropout, 0)**2,axis=1)
            - T.maximum(1+(self.p_y_given_x_dropout)[T.arange(y.shape[0]), y],0)**2))
            + L1_reg*self.L1rec + L2_reg*self.L2rec
        return c

    def errors(self, y):
        """Return a float representing the number of errors in the minibatch
        over the total number of examples of the minibatch ; zero one
        loss over the size of the minibatch

        :type y: theano.tensor.TensorType
        :param y: corresponds to a vector that gives for each example the
                  correct label
        """

        # check if y has same dimension of y_pred
        if y.ndim != self.y_pred.ndim:
            raise TypeError('y should have the same shape as self.y_pred',
                ('y', y.type, 'y_pred', self.y_pred.type))
        # check if y is of the correct datatype
        if y.dtype.startswith('int'):
            # the T.neq operator returns a vector of 0s and 1s, where 1
            # represents a mistake in prediction
            return T.mean(T.neq(self.y_pred, y))
        else:
            raise NotImplementedError()

